package net.lz1998.pbbot.utils;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.IntStream;

public class MeiLvPageProcessor implements PageProcessor{

	//重试3次，每次请求休眠100毫秒，设置超时时间为10000毫秒
	private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setTimeOut(10000);

	private AtomicInteger two = new AtomicInteger(0);

	@Override
	public void process(Page page) {
		Html html = page.getHtml();

		//标题
		List<String> title = html.xpath("//div[@class='top clearfix']/h2/a/text()").all();
		page.putField("title",title);

		//图片
		List<String> img = html.xpath("//div[@class='imagebox']/a").links().all();
		page.putField("img",img);

		//分页图片数据
			IntStream.rangeClosed(2,5).forEach(i->{
			page.addTargetRequest("http://xianrenxz.com/category/gif/jcdt/page/"+i);
		});

		if (page.getResultItems().get("title") == null) {
			page.setSkip(true);

		}
	}

	@Override
	public Site getSite() {
		return site;
	}

	public static void main(String[] args) {
		long startTime= System.currentTimeMillis();
		Spider.create(new MeiLvPageProcessor()).addUrl("http://xianrenxz.com/category/gif/jcdt/page/1")
		.addPipeline(new MeiLvPipeline())
		.thread(16).run();
		long endtTime= System.currentTimeMillis();
		System.out.println("图片下载完成，总花时:"+(endtTime-startTime));

	}
}

